package com.coderfans.datacsdn.parse;

import com.coderfans.datagithub.ConsolePipeline;
import com.coderfans.redis.RedisConfig;
import com.coderfans.redis.impl.RedisCacheClient;
import redis.clients.jedis.JedisPool;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;

import java.util.List;

/**
 * Created by ${fanchunshuai} on 2017-7-13.
 *
 * @version: V1.0
 * @Desc:  博客网页数据抽取
 * @Copyright (c) 2017 58到家-版权所有
 */
public class ParseUserBlog implements PageProcessor {
    //步骤一：抓取网站的相关配置 ： 包括编码 抓取间隔 重试次数等
    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);
    private JedisPool jedisPool = RedisConfig.generatJedisPool();
    private RedisCacheClient setRedis = new RedisCacheClient(jedisPool);

    @Override
    public void process(Page page) {
        page.addTargetRequests(page.getHtml().links().regex("(http://blog.csdn.net/itermeng/article/details/74977888)").all());
        Html html = page.getHtml();
        //博客主题
        String title = html.xpath("//div[@id='article_details']//div[@class='article_title']/h1/span/a/text()").get();
        //原创标识符
        String isOwner = html.xpath("//div[@id='article_details']//div[@class='article_title']/span/@class").get();
        //博客标签
        List<String> tagList = html.xpath("//div[@class='article_manage']//div[@class='article_l']/span//a/text()").all();
        //博客分类
        List<String> typeList = html.xpath("//div[@class='category clearfix']//div[@class='category_r']/label/span/text()").all();
        //发布时间
        String createTime = html.xpath("//div[@id='article_details']/div[@class='article_manage clearfix']/div[@class='article_r']/span/text()").get();
        //当前阅读量
        String readNum = html.xpath("//div[@id='article_details']/div[@class='article_manage clearfix']/div[@class='article_r']/span[@class='link_view']/text()").get();
        //评论数量
        String commNum = html.xpath("//div[@id='article_details']/div[@class='article_manage clearfix']/div[@class='article_r']/span[@class='link_comments']/text()").get();
        //博客内容
        String main = html.xpath("//div[@id='article_details']/div[@class='article_content']").get();
        //赞的数量
        String zanNum = html.xpath("//div[@id='article_details']/div[@id='digg']/dl[@id='btnDigg']/dd/text()").get();
        //踩的数量
        String caiNum = html.xpath("//div[@id='article_details']/div[@id='digg']/dl[@id='btnBury']/dd/text()").get();
        //相关性文章
        List<String> similarList = html.xpath("//div[@class='similar_article']/div[@class='similar_c']//a/@href").all();

        //文章评论者
        List<String> commUserLinkList = html.xpath("//div[@class='comment_class']/div[@id='comment_list']").all();

        System.out.println(title);
        System.out.println(isOwner);
        System.out.println(tagList);
        System.out.println(typeList);
        System.out.println(createTime);
        System.out.println("read= "+readNum);
        System.out.println(commNum);
        System.out.println(main);
        System.out.println("zan = "+zanNum);
        System.out.println("cai = "+caiNum);
        System.out.println(similarList);
        System.out.println(commUserLinkList);
    }

    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        Spider.create(new ParseUserBlog()).addPipeline(new ConsolePipeline())
                .addUrl("http://blog.csdn.net/itermeng/article/details/74977888")
                .thread(1).run();
    }
}

